{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Uncertainty estimation for regression\n",
    "We would demonstrate how to estimate the uncertainty for a regression task. In this case we treat uncertainty as a standard deviation for test data points.\n",
    "As an example dataset we take the kinemtic movement data from UCI database and would estimate the uncertainty prediction with log likelihood metric"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import r2_score \n",
    "\n",
    "from alpaca.ue import MCDUE\n",
    "from alpaca.utils.datasets.builder import build_dataset\n",
    "from alpaca.utils.ue_metrics import ndcg, uq_ll\n",
    "from alpaca.ue.masks import BasicBernoulliMask, DecorrelationMask, LeverageScoreMask\n",
    "from alpaca.utils import model_builder\n",
    "import alpaca.nn as ann"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the dataset\n",
    "The alpaca library has a few regression dataset provided (these datasets often used in the related scientific papers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = build_dataset('kin8nm', val_split=1_000)\n",
    "x_train, y_train = dataset.dataset('train')\n",
    "x_val, y_val = dataset.dataset('val')\n",
    "x_train.shape, y_val.shape\n",
    "train_ds = TensorDataset(torch.FloatTensor(x_train), torch.FloatTensor(y_train))\n",
    "val_ds = TensorDataset(torch.FloatTensor(x_val), torch.FloatTensor(y_val))\n",
    "train_loader = DataLoader(train_ds, batch_size=512)\n",
    "val_loader = DataLoader(val_ds, batch_size=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's build the simple model\n",
    "We'll replace common nn.Dropout layer with ann.Dropout from alpaca.\n",
    "Alpaca version allow to switch on the dropout during inference without worrying other \"training\" layers, like batch norm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, input_size, base_size=64, dropout_rate=0., dropout_mask=None):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(input_size, 4*base_size),\n",
    "            nn.CELU(),\n",
    "\n",
    "            nn.Linear(4*base_size, 2*base_size),\n",
    "            ann.Dropout(dropout_rate, dropout_mask),\n",
    "            nn.CELU(),\n",
    "\n",
    "            nn.Linear(2*base_size, 1*base_size),\n",
    "            ann.Dropout(dropout_rate, dropout_mask),\n",
    "            nn.CELU(),\n",
    "\n",
    "            nn.Linear(base_size, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.net(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model\n",
    "model = MLP(input_size=8, dropout_rate=0.1, dropout_mask=BasicBernoulliMask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
<<<<<<< HEAD:examples/01_regression_uq.ipynb
      "Train loss on last batch 0.000422369601437822\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "model.train()\n",
    "for epochs in range(100):\n",
    "    for x_batch, y_batch in train_loader: # Train for one epoch\n",
    "        predictions = model(x_batch)\n",
    "        loss = criterion(predictions, y_batch)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "print('Train loss on last batch', loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2: 0.9217043442551243\n"
     ]
    }
   ],
   "source": [
    "# Check model effectiveness \n",
    "model.eval()\n",
    "x_batch, y_batch = next(iter(val_loader))\n",
    "predictions = model(x_batch).detach().cpu().numpy()\n",
    "print('R2:', r2_score(predictions, y_batch))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimate uncertainty\n",
    "We compare the log likelihood for constant prediction and monte-carlo uncertainty estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
<<<<<<< HEAD:examples/01_regression_uq.ipynb
=======
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uncertainty estimation with MCDUE_regression approach: 100%|██████████| 25/25 [00:00<00:00, 669.88it/s]\n"
     ]
    }
   ],
   "source": [
    "# Calculate uncertainty estimation\n",
    "estimator = MCDUE(model)\n",
    "predictions, estimations = estimator(x_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
>>>>>>> master:examples/regression_uq.ipynb
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
<<<<<<< HEAD:examples/01_regression_uq.ipynb
      "Quality score for const std is  0.27723035\n"
=======
      "Quality score is  0.9002674676766734\n"
>>>>>>> master:examples/regression_uq.ipynb
     ]
    }
   ],
   "source": [
    "# Baseline\n",
    "const_std = np.std(y_val)\n",
    "errors = np.abs(predictions - y_batch.reshape((-1)).numpy())\n",
    "score = uq_ll(errors, np.ones_like(errors) * const_std)\n",
    "print(\"Quality score for const std is \", score)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
<<<<<<< HEAD:examples/01_regression_uq.ipynb
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train()\n",
    "estimator = MCDUE(model, nn_runs=100)"
=======
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from alpaca.utils import model_builder\n",
    "model = model_builder.build_model(model, dropout_rate=0.2, dropout_mask=DecorrelationMask())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLP(\n",
       "  (fcs): ModuleList(\n",
       "    (0): Sequential(\n",
       "      (0): Linear(in_features=8, out_features=256, bias=True)\n",
       "      (1): Sequential()\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): Linear(in_features=256, out_features=128, bias=True)\n",
       "      (1): Dropout(p=0.0, inplace=None)\n",
       "    )\n",
       "    (2): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): Dropout(p=0.0, inplace=None)\n",
       "    )\n",
       "    (3): Sequential(\n",
       "      (0): Linear(in_features=64, out_features=1, bias=True)\n",
       "      (1): Sequential()\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
>>>>>>> master:examples/regression_uq.ipynb
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD:examples/01_regression_uq.ipynb
   "execution_count": 9,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uncertainty estimation with MCDUE_regression approach: 100%|██████████| 100/100 [00:00<00:00, 2998.07it/s]\n"
=======
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 84 is out of bounds for dimension 0 with size 64",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-438bb4cf3607>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Calculate uncertainty estimation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mestimator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCDUE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mestimations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/rasul/Desktop/forks/alpaca/alpaca/ue/mcdue.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, X_pool)\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0mmcd_runs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_pool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mnn_run\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn_runs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn_runs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdesc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/py3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    720\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    723\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    724\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-97ebc194d528>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, dropout_rate, dropout_mask)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfcs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/py3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    720\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    723\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    724\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/py3/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/rasul/Desktop/forks/alpaca/alpaca/nn/modules/dropout.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     43\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muncertainty_mode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout_mask\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                 return input * self.dropout_mask(\n\u001b[0;32m---> 45\u001b[0;31m                     \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout_rate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m                 )\n\u001b[1;32m     47\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/rasul/Desktop/forks/alpaca/alpaca/ue/masks.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, x, dropout_rate, is_train)\u001b[0m\n\u001b[1;32m    161\u001b[0m             \u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minds\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minds\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mdropout_rate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_as\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 84 is out of bounds for dimension 0 with size 64"
>>>>>>> master:examples/regression_uq.ipynb
     ]
    }
   ],
   "source": [
    "predictions, estimations = estimator(x_batch)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
<<<<<<< HEAD:examples/01_regression_uq.ipynb
   "execution_count": 10,
=======
   "execution_count": 17,
   "metadata": {},
>>>>>>> master:examples/regression_uq.ipynb
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quality score for monte-carlo dropout is  0.37827393\n"
     ]
    }
   ],
   "source": [
<<<<<<< HEAD:examples/01_regression_uq.ipynb
    "errors = np.abs(predictions - y_batch.reshape((-1)).numpy())\n",
    "score = uq_ll(np.array(errors), predictions)\n",
    "print(\"Quality score for monte-carlo dropout is \", score)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
=======
    "# Calculate NDCG score for the uncertainty\n",
    "errors = np.abs(estimations - y_batch.reshape((-1)).numpy()) \n",
    "score = ndcg(np.array(errors), estimations)\n",
    "print(\"Quality score is \", score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
>>>>>>> master:examples/regression_uq.ipynb
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
